# -*- coding: utf-8 -*-
"""
Created on Thu Jun 16 13:15:54 2022
Proprocess environmental complaints for LDA

@author: samir
"""
import pandas as pd
import topicmodels
import topicmodels.preprocess
import os
import nltk
nltk.download('wordnet')
from nltk.stem import WordNetLemmatizer
from nltk import pos_tag
from nltk.corpus import wordnet
nltk.download('averaged_perceptron_tagger')
from paths import target_path #importing target_path from paths.py


def main():
    print("Running: LDA Preprocess Lemmatize")

if __name__ == "__main__":
    main()

# Change to relevant directory
if os.path.exists(target_path):
    os.chdir(target_path)
else:
    print(f"Warning: {target_path} does not exist!")


#Import data
data = pd.read_table("Full_Sample/Raw_Data/complaints_with_year.txt", encoding="utf-8")

###Preliminary Cleaning
docsobj = topicmodels.RawDocs(data.IncidentDescription, "long")

#Calculate number of words before before removing any 
all_tokens = [s for d in docsobj.tokens for s in d]
print("number of unique tokens = %d" % len(set(all_tokens)))
print("number of total tokens = %d" % len(all_tokens))

#Token Clean (Remove tokens of length 1)
docsobj.token_clean(1)

#Calculate number of words after token cleaning
all_tokens = [s for d in docsobj.tokens for s in d]
print("number of unique tokens = %d" % len(set(all_tokens)))
print("number of total tokens = %d" % len(all_tokens))

#Remove stopwards
docsobj.stopword_remove("tokens")

#Number of words after removing stopwards
all_tokens = [s for d in docsobj.tokens for s in d]
print("number of unique tokens = %d" % len(set(all_tokens)))
print("number of total tokens = %d" % len(all_tokens))

###Extract tokens from the docsobj
dataframe = pd.DataFrame()
dataframe['IncidentDescription'] = (docsobj.tokens)

###Define function to modify PoS tag so the lemmatizer can read them, returns 
def pos_tagger(nltk_tag):
    if nltk_tag.startswith('J'):
        return wordnet.ADJ
    elif nltk_tag.startswith('V'):
        return wordnet.VERB
    elif nltk_tag.startswith('N'):
        return wordnet.NOUN
    elif nltk_tag.startswith('R'):
        return wordnet.ADV
    else:         
        return wordnet.NOUN #Noun if part of speech is not one of the basic ones 

###Apply PoS tag
dataframe['IncidentDescriptionTagged'] = dataframe['IncidentDescription'].apply(pos_tag) 

###Lemmatize the dataset
dataframe['IncidentDescriptionLemma'] = ""
wnl = WordNetLemmatizer()
for idx, row in dataframe.iterrows():
        for word, tags in dataframe.loc[idx, 'IncidentDescriptionTagged']:
                dataframe.at[idx, 'IncidentDescriptionLemma'] = dataframe.at[idx, 'IncidentDescriptionLemma'] + wnl.lemmatize(word, pos = pos_tagger(tags)) + " "

###Add in identifying information
Info_frame = data.drop('IncidentDescription', axis = 1)
Merged_frame = pd.concat([Info_frame, dataframe], axis = 1)

###Save as csv
Merged_frame.to_csv(r'Full_Sample/Processed_Data/Complaints_lemmatized.csv', index = False)













